1 Effect of UPSTM-Based Decorrelation on Feature Discovery

1.0.1 Loading the libraries

library("FRESA.CAD")
library(readxl)
library(igraph)
library(umap)
library(tsne)
library(entropy)

op <- par(no.readonly = TRUE)
pander::panderOptions('digits', 3)
pander::panderOptions('table.split.table', 400)
pander::panderOptions('keep.trailing.zeros',TRUE)

1.1 Material and Methods

1.2 The Data


dataGI <- as.data.frame(read_excel("~/GitHub/LatentBiomarkers/Data/GI/data.xlsx", sheet = "Sheet1"))
dataGI$ID <- NULL

table(dataGI$V2)
#> 
#>  1  2 
#> 76 76
dataSet1 <- subset(dataGI,V2==1)
class <- dataSet1$V1
dataSet1$V1 <- NULL
dataSet1$V2 <- NULL
colnames(dataSet1) <- paste(colnames(dataSet1),"WL",sep="_")
dataSet2 <- subset(dataGI,V2==2)
dataSet2$V1 <- NULL
dataSet2$V2 <- NULL
colnames(dataSet2) <- paste(colnames(dataSet2),"NBI",sep="_")
dataGI <- cbind(dataSet1,dataSet2)
dataGI$class <- 1*(class > 1)
table(dataGI$class)
#> 
#>  0  1 
#> 21 55

1.2.0.1 Standarize the names for the reporting

studyName <- "GI"
dataframe <- dataGI
outcome <- "class"

TopVariables <- 10

thro <- 0.80
cexheat = 0.15

1.3 Generaring the report

1.3.1 Libraries

Some libraries

library(psych)
library(whitening)
library("vioplot")
library("rpart")

1.3.2 Data specs

pander::pander(c(rows=nrow(dataframe),col=ncol(dataframe)-1))
rows col
76 1396
pander::pander(table(dataframe[,outcome]))
0 1
21 55

varlist <- colnames(dataframe)
varlist <- varlist[varlist != outcome]

largeSet <- length(varlist) > 1500 

1.3.3 Scaling the data

Scaling and removing near zero variance columns and highly co-linear(r>0.99999) columns


  ### Some global cleaning
  sdiszero <- apply(dataframe,2,sd) > 1.0e-16
  dataframe <- dataframe[,sdiszero]

  varlist <- colnames(dataframe)[colnames(dataframe) != outcome]
  tokeep <- c(as.character(correlated_Remove(dataframe,varlist,thr=0.99999)),outcome)
  dataframe <- dataframe[,tokeep]

  varlist <- colnames(dataframe)
  varlist <- varlist[varlist != outcome]
  
  iscontinous <- sapply(apply(dataframe,2,unique),length) >= 5 ## Only variables with enough samples



dataframeScaled <- FRESAScale(dataframe,method="OrderLogit")$scaledData

1.4 The heatmap of the data

numsub <- nrow(dataframe)
if (numsub > 1000) numsub <- 1000


if (!largeSet)
{

  hm <- heatMaps(data=dataframeScaled[1:numsub,],
                 Outcome=outcome,
                 Scale=TRUE,
                 hCluster = "row",
                 xlab="Feature",
                 ylab="Sample",
                 srtCol=45,
                 srtRow=45,
                 cexCol=cexheat,
                 cexRow=cexheat
                 )
  par(op)
}

1.4.0.1 Correlation Matrix of the Data

The heat map of the data


if (!largeSet)
{

  par(cex=0.6,cex.main=0.85,cex.axis=0.7)
  #cormat <- Rfast::cora(as.matrix(dataframe[,varlist]),large=TRUE)
  cormat <- cor(dataframe[,varlist],method="pearson")
  cormat[is.na(cormat)] <- 0
  gplots::heatmap.2(abs(cormat),
                    trace = "none",
  #                  scale = "row",
                    mar = c(5,5),
                    col=rev(heat.colors(5)),
                    main = "Original Correlation",
                    cexRow = cexheat,
                    cexCol = cexheat,
                     srtCol=45,
                     srtRow=45,
                    key.title=NA,
                    key.xlab="|Pearson Correlation|",
                    xlab="Feature", ylab="Feature")
  diag(cormat) <- 0
  print(max(abs(cormat)))
}

[1] 0.9999797

1.5 The decorrelation


DEdataframe <- IDeA(dataframe,verbose=TRUE,thr=thro)
#> 
#>  Included: 725 , Uni p: 0.0003448276 , Outcome-Driven Size: 0 , Base Size: 110 , Rcrit: 0.3808127 
#> 
#> 
 1 <R=1.000,thr=0.900,N=  525>, Top: 72( 66 )[ 1 : 72 Fa= 72 : 0.900 ]( 72 , 281 , 0 ),<|>Tot Used: 353 , Added: 281 , Zero Std: 0 , Max Cor: 1.000
#> 
 2 <R=1.000,thr=0.900,N=  525>, Top: 33( 14 )[ 1 : 33 Fa= 104 : 0.900 ]( 33 , 170 , 72 ),<|>Tot Used: 476 , Added: 170 , Zero Std: 0 , Max Cor: 0.998
#> 
 3 <R=0.998,thr=0.900,N=  525>, Top: 19( 23 )[ 1 : 19 Fa= 122 : 0.900 ]( 19 , 64 , 104 ),<|>Tot Used: 498 , Added: 64 , Zero Std: 0 , Max Cor: 0.989
#> 
 4 <R=0.989,thr=0.900,N=  525>, Top: 3( 10 )[ 1 : 3 Fa= 125 : 0.900 ]( 3 , 14 , 122 ),<|>Tot Used: 498 , Added: 14 , Zero Std: 0 , Max Cor: 0.900
#> 
 5 <R=0.900,thr=0.800,N=  329>, Top: 89( 1 )[ 1 : 89 Fa= 160 : 0.800 ]( 85 , 168 , 125 ),<|>Tot Used: 581 , Added: 168 , Zero Std: 0 , Max Cor: 0.942
#> 
 6 <R=0.942,thr=0.900,N=   10>, Top: 5( 1 )[ 1 : 5 Fa= 161 : 0.900 ]( 5 , 5 , 160 ),<|>Tot Used: 582 , Added: 5 , Zero Std: 0 , Max Cor: 0.894
#> 
 7 <R=0.894,thr=0.800,N=   82>, Top: 33( 2 )[ 1 : 33 Fa= 178 : 0.800 ]( 33 , 42 , 161 ),<|>Tot Used: 612 , Added: 42 , Zero Std: 0 , Max Cor: 0.978
#> 
 8 <R=0.978,thr=0.900,N=    2>, Top: 1( 1 )[ 1 : 1 Fa= 178 : 0.900 ]( 1 , 1 , 178 ),<|>Tot Used: 612 , Added: 1 , Zero Std: 0 , Max Cor: 0.843
#> 
 9 <R=0.843,thr=0.800,N=    6>, Top: 3( 1 )[ 1 : 3 Fa= 180 : 0.800 ]( 3 , 3 , 178 ),<|>Tot Used: 614 , Added: 3 , Zero Std: 0 , Max Cor: 0.799
#> 
 10 <R=0.799,thr=0.800,N=    6>
#> 
 [ 10 ], 0.7979205 Decor Dimension: 614 Nused: 614 . Cor to Base: 384 , ABase: 32 , Outcome Base: 0 
#> 
varlistc <- colnames(DEdataframe)[colnames(DEdataframe) != outcome]

pander::pander(sum(apply(dataframe[,varlist],2,var)))

7.73e+08

pander::pander(sum(apply(DEdataframe[,varlistc],2,var)))

1.42e+08

pander::pander(entropy(discretize(unlist(dataframe[,varlist]), 256)))

0.306

pander::pander(entropy(discretize(unlist(DEdataframe[,varlistc]), 256)))

0.246

1.5.1 The decorrelation matrix


if (!largeSet)
{

  par(cex=0.6,cex.main=0.85,cex.axis=0.7)
  
  UPLTM <- attr(DEdataframe,"UPLTM")
  
  gplots::heatmap.2(1.0*(abs(UPLTM)>0),
                    trace = "none",
                    mar = c(5,5),
                    col=rev(heat.colors(5)),
                    main = "Decorrelation matrix",
                    cexRow = cexheat,
                    cexCol = cexheat,
                   srtCol=45,
                   srtRow=45,
                    key.title=NA,
                    key.xlab="|Beta|>0",
                    xlab="Output Feature", ylab="Input Feature")
  
  par(op)
}

1.6 The heatmap of the decorrelated data

if (!largeSet)
{

  hm <- heatMaps(data=DEdataframe[1:numsub,],
                 Outcome=outcome,
                 Scale=TRUE,
                 hCluster = "row",
                 cexRow = cexheat,
                 cexCol = cexheat,
                 srtCol=45,
                 srtRow=45,
                 xlab="Feature",
                 ylab="Sample")
  par(op)
}

1.7 The correlation matrix after decorrelation

if (!largeSet)
{

  cormat <- cor(DEdataframe[,varlistc],method="pearson")
  cormat[is.na(cormat)] <- 0
  
  gplots::heatmap.2(abs(cormat),
                    trace = "none",
                    mar = c(5,5),
                    col=rev(heat.colors(5)),
                    main = "Correlation after IDeA",
                    cexRow = cexheat,
                    cexCol = cexheat,
                     srtCol=45,
                     srtRow=45,
                    key.title=NA,
                    key.xlab="|Pearson Correlation|",
                    xlab="Feature", ylab="Feature")
  
  par(op)
  diag(cormat) <- 0
  print(max(abs(cormat)))
}

[1] 0.9419448

1.8 U-MAP Visualization of features

1.8.1 The UMAP based on LASSO on Raw Data


if (nrow(dataframe) < 1000)
{
  classes <- unique(dataframe[1:numsub,outcome])
  raincolors <- rainbow(length(classes))
  names(raincolors) <- classes
  datasetframe.umap = umap(scale(dataframe[1:numsub,varlist]),n_components=2)
  plot(datasetframe.umap$layout,xlab="U1",ylab="U2",main="UMAP: Original",t='n')
  text(datasetframe.umap$layout,labels=dataframe[1:numsub,outcome],col=raincolors[dataframe[1:numsub,outcome]+1])
}

1.8.2 The decorralted UMAP

if (nrow(dataframe) < 1000)
{

  datasetframe.umap = umap(scale(DEdataframe[1:numsub,varlistc]),n_components=2)
  plot(datasetframe.umap$layout,xlab="U1",ylab="U2",main="UMAP: After IDeA",t='n')
  text(datasetframe.umap$layout,labels=DEdataframe[1:numsub,outcome],col=raincolors[DEdataframe[1:numsub,outcome]+1])
}

1.9 Univariate Analysis

1.9.1 Univariate



univarRAW <- uniRankVar(varlist,
               paste(outcome,"~1"),
               outcome,
               dataframe,
               rankingTest="AUC")

100 : V102_WL 200 : V288_WL 300 : V535_WL 400 : V635_WL 500 : V37_NBI
600 : V137_NBI 700 : V470_NBI




univarDe <- uniRankVar(varlistc,
               paste(outcome,"~1"),
               outcome,
               DEdataframe,
               rankingTest="AUC",
               )

100 : La_V102_WL 200 : La_V288_WL 300 : La_V535_WL 400 : La_V635_WL 500 : La_V37_NBI
600 : La_V137_NBI 700 : La_V470_NBI

1.9.2 Final Table


univariate_columns <- c("caseMean","caseStd","controlMean","controlStd","controlKSP","ROCAUC")

##top variables
topvar <- c(1:length(varlist)) <= TopVariables
tableRaw <- univarRAW$orderframe[topvar,univariate_columns]
pander::pander(tableRaw)
  caseMean caseStd controlMean controlStd controlKSP ROCAUC
V172_WL 3.55e+03 1.78e+03 1046.667 537.2409 0.718095 0.933
V220_NBI 2.01e+02 1.20e+02 51.524 27.8220 0.747592 0.929
V220_WL 1.96e+02 1.07e+02 52.381 42.7370 0.097268 0.927
V477_NBI 6.18e-02 2.98e-02 0.149 0.1717 0.000358 0.925
V169_NBI 1.26e+03 8.24e+02 346.619 198.5476 0.350000 0.920
V196_NBI 4.52e+02 2.51e+02 134.238 66.3226 0.410564 0.920
V182_NBI 3.44e+02 2.17e+02 95.190 48.8412 0.793090 0.915
V470_NBI 3.79e-01 1.34e-01 0.188 0.0682 0.948083 0.913
V182_WL 3.17e+02 1.69e+02 96.476 87.3691 0.142781 0.912
V474_NBI 3.40e+00 3.13e-01 2.680 0.5481 0.222068 0.912


topLAvar <- univarDe$orderframe$Name[str_detect(univarDe$orderframe$Name,"La_")]
topLAvar <- unique(c(univarDe$orderframe$Name[topvar],topLAvar[1:as.integer(TopVariables/2)]))
finalTable <- univarDe$orderframe[topLAvar,univariate_columns]


pander::pander(finalTable)
  caseMean caseStd controlMean controlStd controlKSP ROCAUC
V474_NBI 3.40e+00 3.13e-01 2.67965 5.48e-01 0.2221 0.912
V474_WL 3.19e+00 4.57e-01 2.35952 5.29e-01 0.9972 0.882
V481_NBI 4.30e-01 1.32e-01 0.24769 7.51e-02 0.4327 0.879
V4_WL 1.67e+03 9.90e+02 600.08714 4.77e+02 0.0868 0.874
V178_WL 3.20e+02 1.90e+02 124.47619 1.15e+02 0.0797 0.869
La_V47_WL 3.08e-04 8.73e-04 0.00127 7.88e-04 0.7220 0.866
V473_NBI 1.22e-01 4.19e-02 0.21230 1.67e-01 0.0188 0.865
La_V216_NBI -4.14e+01 5.36e+01 10.32451 3.87e+01 0.1341 0.859
V192_WL 3.89e+02 1.94e+02 165.66667 8.74e+01 0.7805 0.855
La_V27_NBI 1.12e-03 8.33e-04 0.00222 8.42e-04 0.7336 0.852
La_V489_WL 9.12e-01 2.72e-02 0.94929 3.01e-02 0.9605 0.830
La_V260_WL -1.57e+01 1.85e+01 -4.19375 7.78e+00 0.4113 0.829

dc <- getLatentCoefficients(DEdataframe)
fscores <- attr(DEdataframe,"fscore")


pander::pander(c(mean=mean(sapply(dc,length)),total=length(dc),fraction=length(dc)/(ncol(dataframe)-1)))
mean total fraction
2.47 529 0.725

theCharformulas <- attr(dc,"LatentCharFormulas")


finalTable <- rbind(finalTable,tableRaw[topvar[!(topvar %in% topLAvar)],univariate_columns])


orgnamez <- rownames(finalTable)
orgnamez <- str_remove_all(orgnamez,"La_")
finalTable$RAWAUC <- univarRAW$orderframe[orgnamez,"ROCAUC"]
finalTable$DecorFormula <- theCharformulas[rownames(finalTable)]
finalTable$fscores <- fscores[rownames(finalTable)]

Final_Columns <- c("DecorFormula","caseMean","caseStd","controlMean","controlStd","controlKSP","ROCAUC","RAWAUC","fscores")

finalTable <- finalTable[order(-finalTable$ROCAUC),]
pander::pander(finalTable[,Final_Columns])
  DecorFormula caseMean caseStd controlMean controlStd controlKSP ROCAUC RAWAUC fscores
V172_WL NA 3.55e+03 1.78e+03 1.05e+03 5.37e+02 0.718095 0.933 0.933 NA
V220_NBI NA 2.01e+02 1.20e+02 5.15e+01 2.78e+01 0.747592 0.929 0.929 NA
V220_WL NA 1.96e+02 1.07e+02 5.24e+01 4.27e+01 0.097268 0.927 0.927 NA
V477_NBI NA 6.18e-02 2.98e-02 1.49e-01 1.72e-01 0.000358 0.925 0.925 NA
V169_NBI NA 1.26e+03 8.24e+02 3.47e+02 1.99e+02 0.350000 0.920 0.920 NA
V196_NBI NA 4.52e+02 2.51e+02 1.34e+02 6.63e+01 0.410564 0.920 0.920 NA
V182_NBI NA 3.44e+02 2.17e+02 9.52e+01 4.88e+01 0.793090 0.915 0.915 NA
V470_NBI NA 3.79e-01 1.34e-01 1.88e-01 6.82e-02 0.948083 0.913 0.913 NA
V474_NBI NA 3.40e+00 3.13e-01 2.68e+00 5.48e-01 0.222068 0.912 0.912 NA
V182_WL NA 3.17e+02 1.69e+02 9.65e+01 8.74e+01 0.142781 0.912 0.912 NA
V474_NBI1 NA 3.40e+00 3.13e-01 2.68e+00 5.48e-01 0.222068 0.912 NA NA
V474_WL NA 3.19e+00 4.57e-01 2.36e+00 5.29e-01 0.997159 0.882 0.882 NA
V481_NBI NA 4.30e-01 1.32e-01 2.48e-01 7.51e-02 0.432665 0.879 0.879 3
V4_WL NA 1.67e+03 9.90e+02 6.00e+02 4.77e+02 0.086777 0.874 0.874 4
V178_WL NA 3.20e+02 1.90e+02 1.24e+02 1.15e+02 0.079732 0.869 0.869 12
La_V47_WL + V47_WL - (0.482)V69_WL 3.08e-04 8.73e-04 1.27e-03 7.88e-04 0.722031 0.866 0.492 -1
V473_NBI NA 1.22e-01 4.19e-02 2.12e-01 1.67e-01 0.018789 0.865 0.865 2
La_V216_NBI - (0.103)V172_NBI - (1.028)V214_NBI + V216_NBI -4.14e+01 5.36e+01 1.03e+01 3.87e+01 0.134150 0.859 0.865 -1
V192_WL NA 3.89e+02 1.94e+02 1.66e+02 8.74e+01 0.780483 0.855 0.855 21
La_V27_NBI + V27_NBI - (0.262)V71_NBI 1.12e-03 8.33e-04 2.22e-03 8.42e-04 0.733561 0.852 0.588 0
La_V489_WL + V489_WL + (0.128)V497_WL 9.12e-01 2.72e-02 9.49e-01 3.01e-02 0.960510 0.830 0.792 1
La_V260_WL - (0.134)V178_WL + V260_WL -1.57e+01 1.85e+01 -4.19e+00 7.78e+00 0.411300 0.829 0.791 1

1.10 Comparing IDeA vs PCA vs EFA

1.10.1 PCA

featuresnames <- colnames(dataframe)[colnames(dataframe) != outcome]
pc <- prcomp(dataframe[,iscontinous],center = TRUE,scale. = TRUE)   #principal components
predPCA <- predict(pc,dataframe[,iscontinous])
PCAdataframe <- as.data.frame(cbind(predPCA,dataframe[,!iscontinous]))
colnames(PCAdataframe) <- c(colnames(predPCA),colnames(dataframe)[!iscontinous]) 
#plot(PCAdataframe[,colnames(PCAdataframe)!=outcome],col=dataframe[,outcome],cex=0.65,cex.lab=0.5,cex.axis=0.75,cex.sub=0.5,cex.main=0.75)

#pander::pander(pc$rotation)


PCACor <- cor(PCAdataframe[,colnames(PCAdataframe) != outcome])


  gplots::heatmap.2(abs(PCACor),
                    trace = "none",
  #                  scale = "row",
                    mar = c(5,5),
                    col=rev(heat.colors(5)),
                    main = "PCA Correlation",
                    cexRow = 0.5,
                    cexCol = 0.5,
                     srtCol=45,
                     srtRow= -45,
                    key.title=NA,
                    key.xlab="Pearson Correlation",
                    xlab="Feature", ylab="Feature")

1.10.2 EFA


EFAdataframe <- dataframeScaled

if (length(iscontinous) < 2000)
{
  topred <- min(length(iscontinous),nrow(dataframeScaled),ncol(predPCA)/2)
  if (topred < 2) topred <- 2
  
  uls <- fa(dataframeScaled[,iscontinous],nfactors=topred,rotate="varimax",warnings=FALSE)  # EFA analysis
  predEFA <- predict(uls,dataframeScaled[,iscontinous])
  EFAdataframe <- as.data.frame(cbind(predEFA,dataframeScaled[,!iscontinous]))
  colnames(EFAdataframe) <- c(colnames(predEFA),colnames(dataframeScaled)[!iscontinous]) 


  
  EFACor <- cor(EFAdataframe[,colnames(EFAdataframe) != outcome])
  
  
    gplots::heatmap.2(abs(EFACor),
                      trace = "none",
    #                  scale = "row",
                      mar = c(5,5),
                      col=rev(heat.colors(5)),
                      main = "EFA Correlation",
                      cexRow = 0.5,
                      cexCol = 0.5,
                       srtCol=45,
                       srtRow= -45,
                      key.title=NA,
                      key.xlab="Pearson Correlation",
                      xlab="Feature", ylab="Feature")
}

1.11 Effect on CAR modeling

par(op)
par(xpd = TRUE)
dataframe[,outcome] <- factor(dataframe[,outcome])
rawmodel <- rpart(paste(outcome,"~."),dataframe,control=rpart.control(maxdepth=3))
pr <- predict(rawmodel,dataframe,type = "class")

  ptab <- list(er="Error",detail=matrix(nrow=6,ncol=1))
  if (length(unique(pr))>1)
  {
    plot(rawmodel,main="Raw",branch=0.5,uniform = TRUE,compress = TRUE,margin=0.1)
    text(rawmodel, use.n = TRUE,cex=0.75)
    ptab <- epiR::epi.tests(table(pr==0,dataframe[,outcome]==0))
  }


pander::pander(table(dataframe[,outcome],pr))
  0 1
0 17 4
1 3 52
pander::pander(ptab$detail[c(5,3,4,6),])
  statistic est lower upper
5 diag.ac 0.908 0.819 0.962
3 se 0.945 0.849 0.989
4 sp 0.810 0.581 0.946
6 diag.or 73.667 14.963 362.674

par(op)
par(xpd = TRUE)
DEdataframe[,outcome] <- factor(DEdataframe[,outcome])
IDeAmodel <- rpart(paste(outcome,"~."),DEdataframe,control=rpart.control(maxdepth=3))
pr <- predict(IDeAmodel,DEdataframe,type = "class")

  ptab <- list(er="Error",detail=matrix(nrow=6,ncol=1))
  if (length(unique(pr))>1)
  {
    plot(IDeAmodel,main="IDeA",branch=0.5,uniform = TRUE,compress = TRUE,margin=0.1)
    text(IDeAmodel, use.n = TRUE,cex=0.75)
    ptab <- epiR::epi.tests(table(pr==0,DEdataframe[,outcome]==0))
  }

pander::pander(table(DEdataframe[,outcome],pr))
  0 1
0 15 6
1 0 55
pander::pander(ptab$detail[c(5,3,4,6),])
  statistic est lower upper
5 diag.ac 0.921 0.836 0.970
3 se 1.000 0.935 1.000
4 sp 0.714 0.478 0.887
6 diag.or Inf NA Inf

par(op)
par(xpd = TRUE)
PCAdataframe[,outcome] <- factor(PCAdataframe[,outcome])
PCAmodel <- rpart(paste(outcome,"~."),PCAdataframe,control=rpart.control(maxdepth=3))
pr <- predict(PCAmodel,PCAdataframe,type = "class")
ptab <- list(er="Error",detail=matrix(nrow=6,ncol=1))
if (length(unique(pr))>1)
{
  plot(PCAmodel,main="PCA",branch=0.5,uniform = TRUE,compress = TRUE,margin=0.1)
  text(PCAmodel, use.n = TRUE,cex=0.75)
  ptab <- epiR::epi.tests(table(pr==0,PCAdataframe[,outcome]==0))
}

pander::pander(table(PCAdataframe[,outcome],pr))
  0 1
0 15 6
1 2 53
pander::pander(ptab$detail[c(5,3,4,6),])
  statistic est lower upper
5 diag.ac 0.895 0.803 0.953
3 se 0.964 0.875 0.996
4 sp 0.714 0.478 0.887
6 diag.or 66.250 12.104 362.601


par(op)

1.11.1 EFA


  EFAdataframe[,outcome] <- factor(EFAdataframe[,outcome])
  EFAmodel <- rpart(paste(outcome,"~."),EFAdataframe,control=rpart.control(maxdepth=3))
  pr <- predict(EFAmodel,EFAdataframe,type = "class")
  
  ptab <- list(er="Error",detail=matrix(nrow=6,ncol=1))
  if (length(unique(pr))>1)
  {
    plot(EFAmodel,main="EFA",branch=0.5,uniform = TRUE,compress = TRUE,margin=0.1)
    text(EFAmodel, use.n = TRUE,cex=0.75)
    ptab <- epiR::epi.tests(table(pr==0,EFAdataframe[,outcome]==0))
  }


  pander::pander(table(EFAdataframe[,outcome],pr))
  0 1
0 16 5
1 0 55
  pander::pander(ptab$detail[c(5,3,4,6),])
  statistic est lower upper
5 diag.ac 0.934 0.853 0.978
3 se 1.000 0.935 1.000
4 sp 0.762 0.528 0.918
6 diag.or Inf NA Inf
  par(op)